/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 *    ClassifierSplitEvaluator.java
 *    Copyright (C) 1999 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.experiment;

import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.classifiers.rules.ZeroR;
import weka.core.AdditionalMeasureProducer;
import weka.core.Attribute;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Option;
import weka.core.OptionHandler;
import weka.core.RevisionHandler;
import weka.core.RevisionUtils;
import weka.core.Summarizable;
import weka.core.Utils;

import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.ObjectStreamClass;
import java.io.Serializable;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.Enumeration;
import java.util.Vector;

/**
 * <!-- globalinfo-start --> A SplitEvaluator that produces results for a
 * classification scheme on a nominal class attribute.
 * <p/>
 * <!-- globalinfo-end -->
 * 
 * <!-- options-start --> Valid options are:
 * <p/>
 * 
 * <pre>
 * -W &lt;class name&gt;
 *  The full class name of the classifier.
 *  eg: weka.classifiers.bayes.NaiveBayes
 * </pre>
 * 
 * <pre>
 * -C &lt;index&gt;
 *  The index of the class for which IR statistics
 *  are to be output. (default 1)
 * </pre>
 * 
 * <pre>
 * -I &lt;index&gt;
 *  The index of an attribute to output in the
 *  results. This attribute should identify an
 *  instance in order to know which instances are
 *  in the test set of a cross validation. if 0
 *  no output (default 0).
 * </pre>
 * 
 * <pre>
 * -P
 *  Add target and prediction columns to the result
 *  for each fold.
 * </pre>
 * 
 * <pre>
 * Options specific to classifier weka.classifiers.rules.ZeroR:
 * </pre>
 * 
 * <pre>
 * -D
 *  If set, classifier is run in debug mode and
 *  may output additional info to the console
 * </pre>
 * 
 * <!-- options-end -->
 * 
 * All options after -- will be passed to the classifier.
 * 
 * @author Len Trigg (trigg@cs.waikato.ac.nz)
 * @version $Revision: 7513 $
 */
public class ClassifierSplitEvaluator implements SplitEvaluator, OptionHandler,
		AdditionalMeasureProducer, RevisionHandler {

	/** for serialization */
	static final long serialVersionUID = -8511241602760467265L;

	/** The template classifier */
	protected Classifier m_Template = new ZeroR();

	/** The classifier used for evaluation */
	protected Classifier m_Classifier;

	/** The names of any additional measures to look for in SplitEvaluators */
	protected String[] m_AdditionalMeasures = null;

	/**
	 * Array of booleans corresponding to the measures in m_AdditionalMeasures
	 * indicating which of the AdditionalMeasures the current classifier can
	 * produce
	 */
	protected boolean[] m_doesProduce = null;

	/**
	 * The number of additional measures that need to be filled in after taking
	 * into account column constraints imposed by the final destination for
	 * results
	 */
	protected int m_numberAdditionalMeasures = 0;

	/** Holds the statistics for the most recent application of the classifier */
	protected String m_result = null;

	/** The classifier options (if any) */
	protected String m_ClassifierOptions = "";

	/** The classifier version */
	protected String m_ClassifierVersion = "";

	/** The length of a key */
	private static final int KEY_SIZE = 3;

	/** The length of a result */
	private static final int RESULT_SIZE = 28;

	/** The number of IR statistics */
	private static final int NUM_IR_STATISTICS = 14;

	/** The number of averaged IR statistics */
	private static final int NUM_WEIGHTED_IR_STATISTICS = 8;

	/** Class index for information retrieval statistics (default 0) */
	private int m_IRclass = 0;

	/** Flag for prediction and target columns output. */
	private boolean m_predTargetColumn = false;

	/** Attribute index of instance identifier (default -1) */
	private int m_attID = -1;

	/**
	 * No args constructor.
	 */
	public ClassifierSplitEvaluator() {

		updateOptions();
	}

	/**
	 * Returns a string describing this split evaluator
	 * 
	 * @return a description of the split evaluator suitable for displaying in
	 *         the explorer/experimenter gui
	 */
	public String globalInfo() {
		return " A SplitEvaluator that produces results for a classification "
				+ "scheme on a nominal class attribute.";
	}

	/**
	 * Returns an enumeration describing the available options..
	 * 
	 * @return an enumeration of all the available options.
	 */
	public Enumeration listOptions() {

		Vector newVector = new Vector(4);

		newVector.addElement(new Option(
				"\tThe full class name of the classifier.\n"
						+ "\teg: weka.classifiers.bayes.NaiveBayes", "W", 1,
				"-W <class name>"));
		newVector.addElement(new Option(
				"\tThe index of the class for which IR statistics\n"
						+ "\tare to be output. (default 1)", "C", 1,
				"-C <index>"));
		newVector.addElement(new Option(
				"\tThe index of an attribute to output in the\n"
						+ "\tresults. This attribute should identify an\n"
						+ "\tinstance in order to know which instances are\n"
						+ "\tin the test set of a cross validation. if 0\n"
						+ "\tno output (default 0).", "I", 1, "-I <index>"));
		newVector.addElement(new Option(
				"\tAdd target and prediction columns to the result\n"
						+ "\tfor each fold.", "P", 0, "-P"));

		if ((m_Template != null) && (m_Template instanceof OptionHandler)) {
			newVector.addElement(new Option("", "", 0,
					"\nOptions specific to classifier "
							+ m_Template.getClass().getName() + ":"));
			Enumeration enu = ((OptionHandler) m_Template).listOptions();
			while (enu.hasMoreElements()) {
				newVector.addElement(enu.nextElement());
			}
		}
		return newVector.elements();
	}

	/**
	 * Parses a given list of options.
	 * <p/>
	 * 
	 * <!-- options-start --> Valid options are:
	 * <p/>
	 * 
	 * <pre>
	 * -W &lt;class name&gt;
	 *  The full class name of the classifier.
	 *  eg: weka.classifiers.bayes.NaiveBayes
	 * </pre>
	 * 
	 * <pre>
	 * -C &lt;index&gt;
	 *  The index of the class for which IR statistics
	 *  are to be output. (default 1)
	 * </pre>
	 * 
	 * <pre>
	 * -I &lt;index&gt;
	 *  The index of an attribute to output in the
	 *  results. This attribute should identify an
	 *  instance in order to know which instances are
	 *  in the test set of a cross validation. if 0
	 *  no output (default 0).
	 * </pre>
	 * 
	 * <pre>
	 * -P
	 *  Add target and prediction columns to the result
	 *  for each fold.
	 * </pre>
	 * 
	 * <pre>
	 * Options specific to classifier weka.classifiers.rules.ZeroR:
	 * </pre>
	 * 
	 * <pre>
	 * -D
	 *  If set, classifier is run in debug mode and
	 *  may output additional info to the console
	 * </pre>
	 * 
	 * <!-- options-end -->
	 * 
	 * All options after -- will be passed to the classifier.
	 * 
	 * @param options
	 *            the list of options as an array of strings
	 * @throws Exception
	 *             if an option is not supported
	 */
	public void setOptions(String[] options) throws Exception {

		String cName = Utils.getOption('W', options);
		if (cName.length() == 0) {
			throw new Exception("A classifier must be specified with"
					+ " the -W option.");
		}
		// Do it first without options, so if an exception is thrown during
		// the option setting, listOptions will contain options for the actual
		// Classifier.
		setClassifier(Classifier.forName(cName, null));
		if (getClassifier() instanceof OptionHandler) {
			((OptionHandler) getClassifier()).setOptions(Utils
					.partitionOptions(options));
			updateOptions();
		}

		String indexName = Utils.getOption('C', options);
		if (indexName.length() != 0) {
			m_IRclass = (new Integer(indexName)).intValue() - 1;
		} else {
			m_IRclass = 0;
		}

		String attID = Utils.getOption('I', options);
		if (attID.length() != 0) {
			m_attID = (new Integer(attID)).intValue() - 1;
		} else {
			m_attID = -1;
		}

		m_predTargetColumn = Utils.getFlag('P', options);
	}

	/**
	 * Gets the current settings of the Classifier.
	 * 
	 * @return an array of strings suitable for passing to setOptions
	 */
	public String[] getOptions() {

		String[] classifierOptions = new String[0];
		if ((m_Template != null) && (m_Template instanceof OptionHandler)) {
			classifierOptions = ((OptionHandler) m_Template).getOptions();
		}

		String[] options = new String[classifierOptions.length + 8];
		int current = 0;

		if (getClassifier() != null) {
			options[current++] = "-W";
			options[current++] = getClassifier().getClass().getName();
		}
		options[current++] = "-I";
		options[current++] = "" + (m_attID + 1);

		if (getPredTargetColumn())
			options[current++] = "-P";

		options[current++] = "-C";
		options[current++] = "" + (m_IRclass + 1);
		options[current++] = "--";

		System.arraycopy(classifierOptions, 0, options, current,
				classifierOptions.length);
		current += classifierOptions.length;
		while (current < options.length) {
			options[current++] = "";
		}
		return options;
	}

	/**
	 * Set a list of method names for additional measures to look for in
	 * Classifiers. This could contain many measures (of which only a subset may
	 * be produceable by the current Classifier) if an experiment is the type
	 * that iterates over a set of properties.
	 * 
	 * @param additionalMeasures
	 *            a list of method names
	 */
	public void setAdditionalMeasures(String[] additionalMeasures) {
		// System.err.println("ClassifierSplitEvaluator: setting additional measures");
		m_AdditionalMeasures = additionalMeasures;

		// determine which (if any) of the additional measures this classifier
		// can produce
		if (m_AdditionalMeasures != null && m_AdditionalMeasures.length > 0) {
			m_doesProduce = new boolean[m_AdditionalMeasures.length];

			if (m_Template instanceof AdditionalMeasureProducer) {
				Enumeration en = ((AdditionalMeasureProducer) m_Template)
						.enumerateMeasures();
				while (en.hasMoreElements()) {
					String mname = (String) en.nextElement();
					for (int j = 0; j < m_AdditionalMeasures.length; j++) {
						if (mname.compareToIgnoreCase(m_AdditionalMeasures[j]) == 0) {
							m_doesProduce[j] = true;
						}
					}
				}
			}
		} else {
			m_doesProduce = null;
		}
	}

	/**
	 * Returns an enumeration of any additional measure names that might be in
	 * the classifier
	 * 
	 * @return an enumeration of the measure names
	 */
	public Enumeration enumerateMeasures() {
		Vector newVector = new Vector();
		if (m_Template instanceof AdditionalMeasureProducer) {
			Enumeration en = ((AdditionalMeasureProducer) m_Template)
					.enumerateMeasures();
			while (en.hasMoreElements()) {
				String mname = (String) en.nextElement();
				newVector.addElement(mname);
			}
		}
		return newVector.elements();
	}

	/**
	 * Returns the value of the named measure
	 * 
	 * @param additionalMeasureName
	 *            the name of the measure to query for its value
	 * @return the value of the named measure
	 * @throws IllegalArgumentException
	 *             if the named measure is not supported
	 */
	public double getMeasure(String additionalMeasureName) {
		if (m_Template instanceof AdditionalMeasureProducer) {
			if (m_Classifier == null) {
				throw new IllegalArgumentException("ClassifierSplitEvaluator: "
						+ "Can't return result for measure, "
						+ "classifier has not been built yet.");
			}
			return ((AdditionalMeasureProducer) m_Classifier)
					.getMeasure(additionalMeasureName);
		} else {
			throw new IllegalArgumentException("ClassifierSplitEvaluator: "
					+ "Can't return value for : " + additionalMeasureName
					+ ". " + m_Template.getClass().getName() + " "
					+ "is not an AdditionalMeasureProducer");
		}
	}

	/**
	 * Gets the data types of each of the key columns produced for a single run.
	 * The number of key fields must be constant for a given SplitEvaluator.
	 * 
	 * @return an array containing objects of the type of each key column. The
	 *         objects should be Strings, or Doubles.
	 */
	public Object[] getKeyTypes() {

		Object[] keyTypes = new Object[KEY_SIZE];
		keyTypes[0] = "";
		keyTypes[1] = "";
		keyTypes[2] = "";
		return keyTypes;
	}

	/**
	 * Gets the names of each of the key columns produced for a single run. The
	 * number of key fields must be constant for a given SplitEvaluator.
	 * 
	 * @return an array containing the name of each key column
	 */
	public String[] getKeyNames() {

		String[] keyNames = new String[KEY_SIZE];
		keyNames[0] = "Scheme";
		keyNames[1] = "Scheme_options";
		keyNames[2] = "Scheme_version_ID";
		return keyNames;
	}

	/**
	 * Gets the key describing the current SplitEvaluator. For example This may
	 * contain the name of the classifier used for classifier predictive
	 * evaluation. The number of key fields must be constant for a given
	 * SplitEvaluator.
	 * 
	 * @return an array of objects containing the key.
	 */
	public Object[] getKey() {

		Object[] key = new Object[KEY_SIZE];
		key[0] = m_Template.getClass().getName();
		key[1] = m_ClassifierOptions;
		key[2] = m_ClassifierVersion;
		return key;
	}

	/**
	 * Gets the data types of each of the result columns produced for a single
	 * run. The number of result fields must be constant for a given
	 * SplitEvaluator.
	 * 
	 * @return an array containing objects of the type of each result column.
	 *         The objects should be Strings, or Doubles.
	 */
	public Object[] getResultTypes() {
		int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length
				: 0;
		int overall_length = RESULT_SIZE + addm;
		overall_length += NUM_IR_STATISTICS;
		overall_length += NUM_WEIGHTED_IR_STATISTICS;
		if (getAttributeID() >= 0)
			overall_length += 1;
		if (getPredTargetColumn())
			overall_length += 2;
		Object[] resultTypes = new Object[overall_length];
		Double doub = new Double(0);
		int current = 0;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		// IR stats
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		// Weighted IR stats
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		// Timing stats
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		// sizes
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;
		resultTypes[current++] = doub;

		// ID/Targets/Predictions
		if (getAttributeID() >= 0)
			resultTypes[current++] = "";
		if (getPredTargetColumn()) {
			resultTypes[current++] = "";
			resultTypes[current++] = "";
		}

		// Classifier defined extras
		resultTypes[current++] = "";

		// add any additional measures
		for (int i = 0; i < addm; i++) {
			resultTypes[current++] = doub;
		}
		if (current != overall_length) {
			throw new Error("ResultTypes didn't fit RESULT_SIZE");
		}
		return resultTypes;
	}

	/**
	 * Gets the names of each of the result columns produced for a single run.
	 * The number of result fields must be constant for a given SplitEvaluator.
	 * 
	 * @return an array containing the name of each result column
	 */
	public String[] getResultNames() {
		int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length
				: 0;
		int overall_length = RESULT_SIZE + addm;
		overall_length += NUM_IR_STATISTICS;
		overall_length += NUM_WEIGHTED_IR_STATISTICS;
		if (getAttributeID() >= 0)
			overall_length += 1;
		if (getPredTargetColumn())
			overall_length += 2;

		String[] resultNames = new String[overall_length];
		int current = 0;
		resultNames[current++] = "Number_of_training_instances";
		resultNames[current++] = "Number_of_testing_instances";

		// Basic performance stats - right vs wrong
		resultNames[current++] = "Number_correct";
		resultNames[current++] = "Number_incorrect";
		resultNames[current++] = "Number_unclassified";
		resultNames[current++] = "Percent_correct";
		resultNames[current++] = "Percent_incorrect";
		resultNames[current++] = "Percent_unclassified";
		resultNames[current++] = "Kappa_statistic";

		// Sensitive stats - certainty of predictions
		resultNames[current++] = "Mean_absolute_error";
		resultNames[current++] = "Root_mean_squared_error";
		resultNames[current++] = "Relative_absolute_error";
		resultNames[current++] = "Root_relative_squared_error";

		// SF stats
		resultNames[current++] = "SF_prior_entropy";
		resultNames[current++] = "SF_scheme_entropy";
		resultNames[current++] = "SF_entropy_gain";
		resultNames[current++] = "SF_mean_prior_entropy";
		resultNames[current++] = "SF_mean_scheme_entropy";
		resultNames[current++] = "SF_mean_entropy_gain";

		// K&B stats
		resultNames[current++] = "KB_information";
		resultNames[current++] = "KB_mean_information";
		resultNames[current++] = "KB_relative_information";

		// IR stats
		resultNames[current++] = "True_positive_rate";
		resultNames[current++] = "Num_true_positives";
		resultNames[current++] = "False_positive_rate";
		resultNames[current++] = "Num_false_positives";
		resultNames[current++] = "True_negative_rate";
		resultNames[current++] = "Num_true_negatives";
		resultNames[current++] = "False_negative_rate";
		resultNames[current++] = "Num_false_negatives";
		resultNames[current++] = "IR_precision";
		resultNames[current++] = "IR_recall";
		resultNames[current++] = "F_measure";
		resultNames[current++] = "Area_under_ROC";

		// Weighted IR stats
		resultNames[current++] = "Weighted_avg_true_positive_rate";
		resultNames[current++] = "Weighted_avg_false_positive_rate";
		resultNames[current++] = "Weighted_avg_true_negative_rate";
		resultNames[current++] = "Weighted_avg_false_negative_rate";
		resultNames[current++] = "Weighted_avg_IR_precision";
		resultNames[current++] = "Weighted_avg_IR_recall";
		resultNames[current++] = "Weighted_avg_F_measure";
		resultNames[current++] = "Weighted_avg_area_under_ROC";

		// Timing stats
		resultNames[current++] = "Elapsed_Time_training";
		resultNames[current++] = "Elapsed_Time_testing";
		resultNames[current++] = "UserCPU_Time_training";
		resultNames[current++] = "UserCPU_Time_testing";

		// sizes
		resultNames[current++] = "Serialized_Model_Size";
		resultNames[current++] = "Serialized_Train_Set_Size";
		resultNames[current++] = "Serialized_Test_Set_Size";

		// ID/Targets/Predictions
		if (getAttributeID() >= 0)
			resultNames[current++] = "Instance_ID";
		if (getPredTargetColumn()) {
			resultNames[current++] = "Targets";
			resultNames[current++] = "Predictions";
		}

		// Classifier defined extras
		resultNames[current++] = "Summary";
		// add any additional measures
		for (int i = 0; i < addm; i++) {
			resultNames[current++] = m_AdditionalMeasures[i];
		}
		if (current != overall_length) {
			throw new Error("ResultNames didn't fit RESULT_SIZE");
		}
		return resultNames;
	}

	/**
	 * Gets the results for the supplied train and test datasets. Now performs a
	 * deep copy of the classifier before it is built and evaluated (just in
	 * case the classifier is not initialized properly in buildClassifier()).
	 * 
	 * @param train
	 *            the training Instances.
	 * @param test
	 *            the testing Instances.
	 * @return the results stored in an array. The objects stored in the array
	 *         may be Strings, Doubles, or null (for the missing value).
	 * @throws Exception
	 *             if a problem occurs while getting the results
	 */
	public Object[] getResult(Instances train, Instances test) throws Exception {

		if (train.classAttribute().type() != Attribute.NOMINAL) {
			throw new Exception("Class attribute is not nominal!");
		}
		if (m_Template == null) {
			throw new Exception("No classifier has been specified");
		}
		int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length
				: 0;
		int overall_length = RESULT_SIZE + addm;
		overall_length += NUM_IR_STATISTICS;
		overall_length += NUM_WEIGHTED_IR_STATISTICS;
		if (getAttributeID() >= 0)
			overall_length += 1;
		if (getPredTargetColumn())
			overall_length += 2;

		ThreadMXBean thMonitor = ManagementFactory.getThreadMXBean();
		boolean canMeasureCPUTime = thMonitor.isThreadCpuTimeSupported();
		if (canMeasureCPUTime && !thMonitor.isThreadCpuTimeEnabled())
			thMonitor.setThreadCpuTimeEnabled(true);

		Object[] result = new Object[overall_length];
		Evaluation eval = new Evaluation(train);
		m_Classifier = Classifier.makeCopy(m_Template);
		double[] predictions;
		long thID = Thread.currentThread().getId();
		long CPUStartTime = -1, trainCPUTimeElapsed = -1, testCPUTimeElapsed = -1, trainTimeStart, trainTimeElapsed, testTimeStart, testTimeElapsed;

		// training classifier
		trainTimeStart = System.currentTimeMillis();
		if (canMeasureCPUTime)
			CPUStartTime = thMonitor.getThreadUserTime(thID);
		m_Classifier.buildClassifier(train);
		if (canMeasureCPUTime)
			trainCPUTimeElapsed = thMonitor.getThreadUserTime(thID)
					- CPUStartTime;
		trainTimeElapsed = System.currentTimeMillis() - trainTimeStart;

		// testing classifier
		testTimeStart = System.currentTimeMillis();
		if (canMeasureCPUTime)
			CPUStartTime = thMonitor.getThreadUserTime(thID);
		predictions = eval.evaluateModel(m_Classifier, test);
		if (canMeasureCPUTime)
			testCPUTimeElapsed = thMonitor.getThreadUserTime(thID)
					- CPUStartTime;
		testTimeElapsed = System.currentTimeMillis() - testTimeStart;
		thMonitor = null;

		m_result = eval.toSummaryString();
		// The results stored are all per instance -- can be multiplied by the
		// number of instances to get absolute numbers
		int current = 0;
		result[current++] = new Double(train.numInstances());
		result[current++] = new Double(eval.numInstances());
		result[current++] = new Double(eval.correct());
		result[current++] = new Double(eval.incorrect());
		result[current++] = new Double(eval.unclassified());
		result[current++] = new Double(eval.pctCorrect());
		result[current++] = new Double(eval.pctIncorrect());
		result[current++] = new Double(eval.pctUnclassified());
		result[current++] = new Double(eval.kappa());

		result[current++] = new Double(eval.meanAbsoluteError());
		result[current++] = new Double(eval.rootMeanSquaredError());
		result[current++] = new Double(eval.relativeAbsoluteError());
		result[current++] = new Double(eval.rootRelativeSquaredError());

		result[current++] = new Double(eval.SFPriorEntropy());
		result[current++] = new Double(eval.SFSchemeEntropy());
		result[current++] = new Double(eval.SFEntropyGain());
		result[current++] = new Double(eval.SFMeanPriorEntropy());
		result[current++] = new Double(eval.SFMeanSchemeEntropy());
		result[current++] = new Double(eval.SFMeanEntropyGain());

		// K&B stats
		result[current++] = new Double(eval.KBInformation());
		result[current++] = new Double(eval.KBMeanInformation());
		result[current++] = new Double(eval.KBRelativeInformation());

		// IR stats
		result[current++] = new Double(eval.truePositiveRate(m_IRclass));
		result[current++] = new Double(eval.numTruePositives(m_IRclass));
		result[current++] = new Double(eval.falsePositiveRate(m_IRclass));
		result[current++] = new Double(eval.numFalsePositives(m_IRclass));
		result[current++] = new Double(eval.trueNegativeRate(m_IRclass));
		result[current++] = new Double(eval.numTrueNegatives(m_IRclass));
		result[current++] = new Double(eval.falseNegativeRate(m_IRclass));
		result[current++] = new Double(eval.numFalseNegatives(m_IRclass));
		result[current++] = new Double(eval.precision(m_IRclass));
		result[current++] = new Double(eval.recall(m_IRclass));
		result[current++] = new Double(eval.fMeasure(m_IRclass));
		result[current++] = new Double(eval.areaUnderROC(m_IRclass));

		// Weighted IR stats
		result[current++] = new Double(eval.weightedTruePositiveRate());
		result[current++] = new Double(eval.weightedFalsePositiveRate());
		result[current++] = new Double(eval.weightedTrueNegativeRate());
		result[current++] = new Double(eval.weightedFalseNegativeRate());
		result[current++] = new Double(eval.weightedPrecision());
		result[current++] = new Double(eval.weightedRecall());
		result[current++] = new Double(eval.weightedFMeasure());
		result[current++] = new Double(eval.weightedAreaUnderROC());

		// Timing stats
		result[current++] = new Double(trainTimeElapsed / 1000.0);
		result[current++] = new Double(testTimeElapsed / 1000.0);
		if (canMeasureCPUTime) {
			result[current++] = new Double(
					(trainCPUTimeElapsed / 1000000.0) / 1000.0);
			result[current++] = new Double(
					(testCPUTimeElapsed / 1000000.0) / 1000.0);
		} else {
			result[current++] = new Double(Instance.missingValue());
			result[current++] = new Double(Instance.missingValue());
		}

		// sizes
		ByteArrayOutputStream bastream = new ByteArrayOutputStream();
		ObjectOutputStream oostream = new ObjectOutputStream(bastream);
		oostream.writeObject(m_Classifier);
		result[current++] = new Double(bastream.size());
		bastream = new ByteArrayOutputStream();
		oostream = new ObjectOutputStream(bastream);
		oostream.writeObject(train);
		result[current++] = new Double(bastream.size());
		bastream = new ByteArrayOutputStream();
		oostream = new ObjectOutputStream(bastream);
		oostream.writeObject(test);
		result[current++] = new Double(bastream.size());

		// IDs
		if (getAttributeID() >= 0) {
			String idsString = "";
			if (test.attribute(m_attID).isNumeric()) {
				if (test.numInstances() > 0)
					idsString += test.instance(0).value(m_attID);
				for (int i = 1; i < test.numInstances(); i++) {
					idsString += "|" + test.instance(i).value(m_attID);
				}
			} else {
				if (test.numInstances() > 0)
					idsString += test.instance(0).stringValue(m_attID);
				for (int i = 1; i < test.numInstances(); i++) {
					idsString += "|" + test.instance(i).stringValue(m_attID);
				}
			}
			result[current++] = idsString;
		}

		if (getPredTargetColumn()) {
			if (test.classAttribute().isNumeric()) {
				// Targets
				if (test.numInstances() > 0) {
					String targetsString = "";
					targetsString += test.instance(0).value(test.classIndex());
					for (int i = 1; i < test.numInstances(); i++) {
						targetsString += "|"
								+ test.instance(i).value(test.classIndex());
					}
					result[current++] = targetsString;
				}

				// Predictions
				if (predictions.length > 0) {
					String predictionsString = "";
					predictionsString += predictions[0];
					for (int i = 1; i < predictions.length; i++) {
						predictionsString += "|" + predictions[i];
					}
					result[current++] = predictionsString;
				}
			} else {
				// Targets
				if (test.numInstances() > 0) {
					String targetsString = "";
					targetsString += test.instance(0).stringValue(
							test.classIndex());
					for (int i = 1; i < test.numInstances(); i++) {
						targetsString += "|"
								+ test.instance(i).stringValue(
										test.classIndex());
					}
					result[current++] = targetsString;
				}

				// Predictions
				if (predictions.length > 0) {
					String predictionsString = "";
					predictionsString += test.classAttribute().value(
							(int) predictions[0]);
					for (int i = 1; i < predictions.length; i++) {
						predictionsString += "|"
								+ test.classAttribute().value(
										(int) predictions[i]);
					}
					result[current++] = predictionsString;
				}
			}
		}

		if (m_Classifier instanceof Summarizable) {
			result[current++] = ((Summarizable) m_Classifier).toSummaryString();
		} else {
			result[current++] = null;
		}

		for (int i = 0; i < addm; i++) {
			if (m_doesProduce[i]) {
				try {
					double dv = ((AdditionalMeasureProducer) m_Classifier)
							.getMeasure(m_AdditionalMeasures[i]);
					if (!Instance.isMissingValue(dv)) {
						Double value = new Double(dv);
						result[current++] = value;
					} else {
						result[current++] = null;
					}
				} catch (Exception ex) {
					System.err.println(ex);
				}
			} else {
				result[current++] = null;
			}
		}

		if (current != overall_length) {
			throw new Error("Results didn't fit RESULT_SIZE");
		}
		return result;
	}

	/**
	 * Returns the tip text for this property
	 * 
	 * @return tip text for this property suitable for displaying in the
	 *         explorer/experimenter gui
	 */
	public String classifierTipText() {
		return "The classifier to use.";
	}

	/**
	 * Get the value of Classifier.
	 * 
	 * @return Value of Classifier.
	 */
	public Classifier getClassifier() {

		return m_Template;
	}

	/**
	 * Sets the classifier.
	 * 
	 * @param newClassifier
	 *            the new classifier to use.
	 */
	public void setClassifier(Classifier newClassifier) {

		m_Template = newClassifier;
		updateOptions();
	}

	/**
	 * Get the value of ClassForIRStatistics.
	 * 
	 * @return Value of ClassForIRStatistics.
	 */
	public int getClassForIRStatistics() {
		return m_IRclass;
	}

	/**
	 * Set the value of ClassForIRStatistics.
	 * 
	 * @param v
	 *            Value to assign to ClassForIRStatistics.
	 */
	public void setClassForIRStatistics(int v) {
		m_IRclass = v;
	}

	/**
	 * Get the index of Attibute Identifying the instances
	 * 
	 * @return index of outputed Attribute.
	 */
	public int getAttributeID() {
		return m_attID;
	}

	/**
	 * Set the index of Attibute Identifying the instances
	 * 
	 * @param v
	 *            index the attribute to output
	 */
	public void setAttributeID(int v) {
		m_attID = v;
	}

	/**
	 * @return true if the prediction and target columns must be outputed.
	 */
	public boolean getPredTargetColumn() {
		return m_predTargetColumn;
	}

	/**
	 * Set the flag for prediction and target output.
	 * 
	 * @param v
	 *            true if the 2 columns have to be outputed. false otherwise.
	 */
	public void setPredTargetColumn(boolean v) {
		m_predTargetColumn = v;
	}

	/**
	 * Updates the options that the current classifier is using.
	 */
	protected void updateOptions() {

		if (m_Template instanceof OptionHandler) {
			m_ClassifierOptions = Utils
					.joinOptions(((OptionHandler) m_Template).getOptions());
		} else {
			m_ClassifierOptions = "";
		}
		if (m_Template instanceof Serializable) {
			ObjectStreamClass obs = ObjectStreamClass.lookup(m_Template
					.getClass());
			m_ClassifierVersion = "" + obs.getSerialVersionUID();
		} else {
			m_ClassifierVersion = "";
		}
	}

	/**
	 * Set the Classifier to use, given it's class name. A new classifier will
	 * be instantiated.
	 * 
	 * @param newClassifierName
	 *            the Classifier class name.
	 * @throws Exception
	 *             if the class name is invalid.
	 */
	public void setClassifierName(String newClassifierName) throws Exception {

		try {
			setClassifier((Classifier) Class.forName(newClassifierName)
					.newInstance());
		} catch (Exception ex) {
			throw new Exception("Can't find Classifier with class name: "
					+ newClassifierName);
		}
	}

	/**
	 * Gets the raw output from the classifier
	 * 
	 * @return the raw output from th,0e classifier
	 */
	public String getRawResultOutput() {
		StringBuffer result = new StringBuffer();

		if (m_Classifier == null) {
			return "<null> classifier";
		}
		result.append(toString());
		result.append("Classifier model: \n" + m_Classifier.toString() + '\n');

		// append the performance statistics
		if (m_result != null) {
			result.append(m_result);

			if (m_doesProduce != null) {
				for (int i = 0; i < m_doesProduce.length; i++) {
					if (m_doesProduce[i]) {
						try {
							double dv = ((AdditionalMeasureProducer) m_Classifier)
									.getMeasure(m_AdditionalMeasures[i]);
							if (!Instance.isMissingValue(dv)) {
								Double value = new Double(dv);
								result.append(m_AdditionalMeasures[i] + " : "
										+ value + '\n');
							} else {
								result.append(m_AdditionalMeasures[i] + " : "
										+ '?' + '\n');
							}
						} catch (Exception ex) {
							System.err.println(ex);
						}
					}
				}
			}
		}
		return result.toString();
	}

	/**
	 * Returns a text description of the split evaluator.
	 * 
	 * @return a text description of the split evaluator.
	 */
	public String toString() {

		String result = "ClassifierSplitEvaluator: ";
		if (m_Template == null) {
			return result + "<null> classifier";
		}
		return result + m_Template.getClass().getName() + " "
				+ m_ClassifierOptions + "(version " + m_ClassifierVersion + ")";
	}

	/**
	 * Returns the revision string.
	 * 
	 * @return the revision
	 */
	public String getRevision() {
		return RevisionUtils.extract("$Revision: 7513 $");
	}
} // ClassifierSplitEvaluator
